{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#coding:utf-8\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import data_helpers\n",
    "from text_cnn import TextCNN\n",
    "from tensorflow.contrib import learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Parameters:\n",
      "ALLOW_SOFT_PLACEMENT=True\n",
      "BATCH_SIZE=64\n",
      "CHECKPOINT_EVERY=100\n",
      "DEV_SAMPLE_PERCENTAGE=0.1\n",
      "DROPOUT_KEEP_PROB=0.5\n",
      "EMBEDDING_DIM=128\n",
      "EVALUATE_EVERY=100\n",
      "FILTER_SIZES=3,4,5\n",
      "L2_REG_LAMBDA=0.0005\n",
      "LOG_DEVICE_PLACEMENT=False\n",
      "NEGATIVE_DATA_FILE=./data/rt-polaritydata/rt-polarity.neg\n",
      "NUM_CHECKPOINTS=5\n",
      "NUM_EPOCHS=20\n",
      "NUM_FILTERS=64\n",
      "POSITIVE_DATA_FILE=./data/rt-polaritydata/rt-polarity.pos\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Parameters\n",
    "# ==================================================\n",
    "\n",
    "# Data loading params\n",
    "# validation数据集占比\n",
    "tf.flags.DEFINE_float(\"dev_sample_percentage\", .1, \"Percentage of the training data to use for validation\")\n",
    "# 正样本\n",
    "tf.flags.DEFINE_string(\"positive_data_file\", \"./data/rt-polaritydata/rt-polarity.pos\", \"Data source for the positive data.\")\n",
    "# 负样本\n",
    "tf.flags.DEFINE_string(\"negative_data_file\", \"./data/rt-polaritydata/rt-polarity.neg\", \"Data source for the negative data.\")\n",
    "\n",
    "# Model Hyperparameters\n",
    "# 词向量长度\n",
    "tf.flags.DEFINE_integer(\"embedding_dim\", 128, \"Dimensionality of character embedding (default: 128)\")\n",
    "# 卷积核大小\n",
    "tf.flags.DEFINE_string(\"filter_sizes\", \"3,4,5\", \"Comma-separated filter sizes (default: '3,4,5')\")\n",
    "# 每一种卷积核个数\n",
    "tf.flags.DEFINE_integer(\"num_filters\", 64, \"Number of filters per filter size (default: 128)\")\n",
    "# dropout参数\n",
    "tf.flags.DEFINE_float(\"dropout_keep_prob\", 0.5, \"Dropout keep probability (default: 0.5)\")\n",
    "# l2正则化参数\n",
    "tf.flags.DEFINE_float(\"l2_reg_lambda\", 0.0005, \"L2 regularization lambda (default: 0.0)\")\n",
    "\n",
    "# Training parameters\n",
    "# 批次大小\n",
    "tf.flags.DEFINE_integer(\"batch_size\", 64, \"Batch Size (default: 64)\")\n",
    "# 迭代周期\n",
    "tf.flags.DEFINE_integer(\"num_epochs\", 20, \"Number of training epochs (default: 200)\")\n",
    "# 多少step测试一次\n",
    "tf.flags.DEFINE_integer(\"evaluate_every\", 100, \"Evaluate model on dev set after this many steps (default: 100)\")\n",
    "# 多少step保存一次模型\n",
    "tf.flags.DEFINE_integer(\"checkpoint_every\", 100, \"Save model after this many steps (default: 100)\")\n",
    "# 最多保存多少个模型\n",
    "tf.flags.DEFINE_integer(\"num_checkpoints\", 5, \"Number of checkpoints to store (default: 5)\")\n",
    "# Misc Parameters\n",
    "# tensorFlow 会自动选择一个存在并且支持的设备来运行 operation\n",
    "tf.flags.DEFINE_boolean(\"allow_soft_placement\", True, \"Allow device soft device placement\")\n",
    "# 获取你的 operations 和 Tensor 被指派到哪个设备上运行\n",
    "tf.flags.DEFINE_boolean(\"log_device_placement\", False, \"Log placement of ops on devices\")\n",
    "\n",
    "# flags解析\n",
    "FLAGS = tf.flags.FLAGS\n",
    "FLAGS._parse_flags()\n",
    "\n",
    "# 打印所有参数\n",
    "print(\"\\nParameters:\")\n",
    "for attr, value in sorted(FLAGS.__flags.items()):\n",
    "    print(\"{}={}\".format(attr.upper(), value))\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "x_shape: (10662, 56)\n",
      "y_shape: (10662, 2)\n",
      "Vocabulary Size: 18758\n",
      "Train/Dev split: 9596/1066\n",
      "x: [[ 4719    59   182    34   190   804     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [  129  7044   284   146    80     3  1116    58    84  1386   182  1968\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [  146     3   453    88    34     1   190  8338 18328    12  2320  1573\n",
      "   3840  6569 16227   112  1543   246    17  1722  5117     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [ 2718   149  7850   503   343    87  6515   250  2934     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [  343   133     1  1528    34  2691   643 16667   125     1  4435    34\n",
      "    983   740     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]]\n",
      "y: [[1 0]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [1 0]]\n"
     ]
    }
   ],
   "source": [
    "# Data Preparation\n",
    "# ==================================================\n",
    "\n",
    "# Load data\n",
    "print(\"Loading data...\")\n",
    "x_text, y = data_helpers.load_data_and_labels(FLAGS.positive_data_file, FLAGS.negative_data_file)\n",
    "\n",
    "# Build vocabulary\n",
    "# 一行数据最多的词汇数\n",
    "max_document_length = max([len(x.split(\" \")) for x in x_text])\n",
    "vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)\n",
    "x = np.array(list(vocab_processor.fit_transform(x_text)))\n",
    "print(\"x_shape:\",x.shape)\n",
    "print(\"y_shape:\",y.shape)\n",
    "\n",
    "# Randomly shuffle data\n",
    "np.random.seed(10)\n",
    "shuffle_indices = np.random.permutation(np.arange(len(y)))\n",
    "x_shuffled = x[shuffle_indices]\n",
    "y_shuffled = y[shuffle_indices]\n",
    "\n",
    "# Split train/test set\n",
    "# TODO: This is very crude, should use cross-validation\n",
    "# 数据集切分为两部分\n",
    "dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y)))\n",
    "x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]\n",
    "y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]\n",
    "print(\"Vocabulary Size: {:d}\".format(len(vocab_processor.vocabulary_)))\n",
    "print(\"Train/Dev split: {:d}/{:d}\".format(len(y_train), len(y_dev)))\n",
    "\n",
    "print(\"x:\",x_train[0:5])\n",
    "print(\"y:\",y_train[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name embedding/W:0/grad/hist is illegal; using embedding/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name embedding/W:0/grad/sparsity is illegal; using embedding/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/W:0/grad/hist is illegal; using conv-maxpool-3/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/W:0/grad/sparsity is illegal; using conv-maxpool-3/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/b:0/grad/hist is illegal; using conv-maxpool-3/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-3/b:0/grad/sparsity is illegal; using conv-maxpool-3/b_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/W:0/grad/hist is illegal; using conv-maxpool-4/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/W:0/grad/sparsity is illegal; using conv-maxpool-4/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/b:0/grad/hist is illegal; using conv-maxpool-4/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-4/b:0/grad/sparsity is illegal; using conv-maxpool-4/b_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/W:0/grad/hist is illegal; using conv-maxpool-5/W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/W:0/grad/sparsity is illegal; using conv-maxpool-5/W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/b:0/grad/hist is illegal; using conv-maxpool-5/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name conv-maxpool-5/b:0/grad/sparsity is illegal; using conv-maxpool-5/b_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name W:0/grad/hist is illegal; using W_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name W:0/grad/sparsity is illegal; using W_0/grad/sparsity instead.\n",
      "INFO:tensorflow:Summary name output/b:0/grad/hist is illegal; using output/b_0/grad/hist instead.\n",
      "INFO:tensorflow:Summary name output/b:0/grad/sparsity is illegal; using output/b_0/grad/sparsity instead.\n",
      "Writing to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\n",
      "\n",
      "num_batches_per_epoch: 150\n",
      "2017-07-01T10:31:53.076406: step 10, loss 0.832417, acc 0.421875\n",
      "2017-07-01T10:31:54.090102: step 20, loss 0.790208, acc 0.5\n",
      "2017-07-01T10:31:55.114829: step 30, loss 0.911867, acc 0.390625\n",
      "2017-07-01T10:31:56.152589: step 40, loss 0.835447, acc 0.46875\n",
      "2017-07-01T10:31:57.185336: step 50, loss 0.822277, acc 0.4375\n",
      "2017-07-01T10:31:58.197027: step 60, loss 0.73263, acc 0.578125\n",
      "2017-07-01T10:31:59.242809: step 70, loss 0.810601, acc 0.46875\n",
      "2017-07-01T10:32:00.261519: step 80, loss 0.807106, acc 0.453125\n",
      "2017-07-01T10:32:01.256164: step 90, loss 0.72523, acc 0.546875\n",
      "2017-07-01T10:32:02.239781: step 100, loss 0.843065, acc 0.421875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:32:02.418401: step 100, loss 0.719268, acc 0.530957\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-100\n",
      "\n",
      "2017-07-01T10:32:04.467838: step 110, loss 0.788776, acc 0.5\n",
      "2017-07-01T10:32:05.476550: step 120, loss 0.755046, acc 0.5\n",
      "2017-07-01T10:32:06.483208: step 130, loss 0.652462, acc 0.65625\n",
      "2017-07-01T10:32:07.468820: step 140, loss 0.708672, acc 0.578125\n",
      "2017-07-01T10:32:14.479502: step 150, loss 0.730748, acc 0.533333\n",
      "2017-07-01T10:32:15.543315: step 160, loss 0.840153, acc 0.453125\n",
      "2017-07-01T10:32:16.544963: step 170, loss 0.628002, acc 0.65625\n",
      "2017-07-01T10:32:17.545660: step 180, loss 0.688108, acc 0.578125\n",
      "2017-07-01T10:32:18.533253: step 190, loss 0.671867, acc 0.625\n",
      "2017-07-01T10:32:19.528900: step 200, loss 0.68227, acc 0.609375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:32:19.596085: step 200, loss 0.692835, acc 0.572233\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-200\n",
      "\n",
      "2017-07-01T10:32:21.589664: step 210, loss 0.754971, acc 0.515625\n",
      "2017-07-01T10:32:22.575295: step 220, loss 0.701055, acc 0.59375\n",
      "2017-07-01T10:32:23.577953: step 230, loss 0.733688, acc 0.515625\n",
      "2017-07-01T10:32:24.568588: step 240, loss 0.77584, acc 0.453125\n",
      "2017-07-01T10:32:25.573260: step 250, loss 0.705674, acc 0.59375\n",
      "2017-07-01T10:32:26.568909: step 260, loss 0.683846, acc 0.609375\n",
      "2017-07-01T10:32:27.597645: step 270, loss 0.669392, acc 0.625\n",
      "2017-07-01T10:32:28.593328: step 280, loss 0.710503, acc 0.53125\n",
      "2017-07-01T10:32:29.613081: step 290, loss 0.736135, acc 0.53125\n",
      "2017-07-01T10:32:30.614717: step 300, loss 0.722397, acc 0.566667\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:32:30.686909: step 300, loss 0.673554, acc 0.602251\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-300\n",
      "\n",
      "2017-07-01T10:32:32.865306: step 310, loss 0.610526, acc 0.671875\n",
      "2017-07-01T10:32:33.863962: step 320, loss 0.681758, acc 0.578125\n",
      "2017-07-01T10:32:34.854597: step 330, loss 0.679637, acc 0.625\n",
      "2017-07-01T10:32:35.864283: step 340, loss 0.665958, acc 0.671875\n",
      "2017-07-01T10:32:36.861937: step 350, loss 0.603429, acc 0.671875\n",
      "2017-07-01T10:32:37.865606: step 360, loss 0.660844, acc 0.625\n",
      "2017-07-01T10:32:38.901362: step 370, loss 0.69253, acc 0.59375\n",
      "2017-07-01T10:32:39.902040: step 380, loss 0.572463, acc 0.734375\n",
      "2017-07-01T10:32:40.935808: step 390, loss 0.657897, acc 0.640625\n",
      "2017-07-01T10:32:41.926442: step 400, loss 0.624535, acc 0.671875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:32:41.996595: step 400, loss 0.660723, acc 0.621013\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-400\n",
      "\n",
      "2017-07-01T10:32:44.097656: step 410, loss 0.605204, acc 0.625\n",
      "2017-07-01T10:32:45.142466: step 420, loss 0.72156, acc 0.578125\n",
      "2017-07-01T10:32:46.180196: step 430, loss 0.715314, acc 0.578125\n",
      "2017-07-01T10:32:47.194895: step 440, loss 0.655435, acc 0.640625\n",
      "2017-07-01T10:32:48.226668: step 450, loss 0.685796, acc 0.583333\n",
      "2017-07-01T10:32:49.254406: step 460, loss 0.669771, acc 0.640625\n",
      "2017-07-01T10:32:50.270146: step 470, loss 0.688539, acc 0.609375\n",
      "2017-07-01T10:32:51.324952: step 480, loss 0.579967, acc 0.703125\n",
      "2017-07-01T10:32:52.380760: step 490, loss 0.66706, acc 0.609375\n",
      "2017-07-01T10:32:53.427545: step 500, loss 0.708879, acc 0.59375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:32:53.500740: step 500, loss 0.652448, acc 0.635085\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-500\n",
      "\n",
      "2017-07-01T10:32:55.608346: step 510, loss 0.714491, acc 0.578125\n",
      "2017-07-01T10:32:56.620037: step 520, loss 0.574536, acc 0.734375\n",
      "2017-07-01T10:32:57.633735: step 530, loss 0.54987, acc 0.765625\n",
      "2017-07-01T10:32:58.627407: step 540, loss 0.5927, acc 0.703125\n",
      "2017-07-01T10:32:59.629099: step 550, loss 0.589539, acc 0.71875\n",
      "2017-07-01T10:33:00.638771: step 560, loss 0.63446, acc 0.640625\n",
      "2017-07-01T10:33:01.637414: step 570, loss 0.607368, acc 0.671875\n",
      "2017-07-01T10:33:02.685203: step 580, loss 0.736623, acc 0.546875\n",
      "2017-07-01T10:33:03.715977: step 590, loss 0.590118, acc 0.734375\n",
      "2017-07-01T10:33:04.719613: step 600, loss 0.580801, acc 0.7\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:33:04.789799: step 600, loss 0.638314, acc 0.644465\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-600\n",
      "\n",
      "2017-07-01T10:33:06.845973: step 610, loss 0.565022, acc 0.734375\n",
      "2017-07-01T10:33:07.884737: step 620, loss 0.580477, acc 0.6875\n",
      "2017-07-01T10:33:08.913473: step 630, loss 0.642248, acc 0.65625\n",
      "2017-07-01T10:33:09.962263: step 640, loss 0.534662, acc 0.78125\n",
      "2017-07-01T10:33:11.001026: step 650, loss 0.521354, acc 0.78125\n",
      "2017-07-01T10:33:12.032802: step 660, loss 0.558562, acc 0.734375\n",
      "2017-07-01T10:33:13.070531: step 670, loss 0.571718, acc 0.734375\n",
      "2017-07-01T10:33:14.084228: step 680, loss 0.532051, acc 0.796875\n",
      "2017-07-01T10:33:15.075892: step 690, loss 0.561444, acc 0.75\n",
      "2017-07-01T10:33:16.061486: step 700, loss 0.608232, acc 0.71875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:33:16.131685: step 700, loss 0.628346, acc 0.658537\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-700\n",
      "\n",
      "2017-07-01T10:33:18.353532: step 710, loss 0.638634, acc 0.6875\n",
      "2017-07-01T10:33:19.347176: step 720, loss 0.605652, acc 0.671875\n",
      "2017-07-01T10:33:20.333800: step 730, loss 0.630459, acc 0.65625\n",
      "2017-07-01T10:33:21.330482: step 740, loss 0.551783, acc 0.765625\n",
      "2017-07-01T10:33:22.323139: step 750, loss 0.479402, acc 0.833333\n",
      "2017-07-01T10:33:23.321748: step 760, loss 0.542666, acc 0.765625\n",
      "2017-07-01T10:33:24.338499: step 770, loss 0.586868, acc 0.71875\n",
      "2017-07-01T10:33:25.344160: step 780, loss 0.546427, acc 0.75\n",
      "2017-07-01T10:33:26.333760: step 790, loss 0.466141, acc 0.84375\n",
      "2017-07-01T10:33:27.320385: step 800, loss 0.591054, acc 0.65625\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:33:27.390571: step 800, loss 0.635547, acc 0.661351\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-800\n",
      "\n",
      "2017-07-01T10:33:29.469464: step 810, loss 0.50629, acc 0.8125\n",
      "2017-07-01T10:33:30.456088: step 820, loss 0.650442, acc 0.65625\n",
      "2017-07-01T10:33:31.455780: step 830, loss 0.508635, acc 0.796875\n",
      "2017-07-01T10:33:32.456445: step 840, loss 0.569102, acc 0.71875\n",
      "2017-07-01T10:33:33.445072: step 850, loss 0.542133, acc 0.765625\n",
      "2017-07-01T10:33:34.488817: step 860, loss 0.441672, acc 0.84375\n",
      "2017-07-01T10:33:35.539610: step 870, loss 0.542081, acc 0.78125\n",
      "2017-07-01T10:33:36.556316: step 880, loss 0.439379, acc 0.84375\n",
      "2017-07-01T10:33:37.587057: step 890, loss 0.529645, acc 0.78125\n",
      "2017-07-01T10:33:38.614791: step 900, loss 0.556279, acc 0.733333\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:33:38.687985: step 900, loss 0.610862, acc 0.67636\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-900\n",
      "\n",
      "2017-07-01T10:33:40.731485: step 910, loss 0.550386, acc 0.75\n",
      "2017-07-01T10:33:41.712093: step 920, loss 0.626616, acc 0.65625\n",
      "2017-07-01T10:33:42.732840: step 930, loss 0.477498, acc 0.828125\n",
      "2017-07-01T10:33:43.817706: step 940, loss 0.553352, acc 0.75\n",
      "2017-07-01T10:33:44.831391: step 950, loss 0.44541, acc 0.84375\n",
      "2017-07-01T10:33:45.832081: step 960, loss 0.445571, acc 0.890625\n",
      "2017-07-01T10:33:46.844746: step 970, loss 0.471181, acc 0.828125\n",
      "2017-07-01T10:33:47.870476: step 980, loss 0.479194, acc 0.8125\n",
      "2017-07-01T10:33:48.928289: step 990, loss 0.562513, acc 0.71875\n",
      "2017-07-01T10:33:50.001143: step 1000, loss 0.542606, acc 0.75\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:33:50.076347: step 1000, loss 0.604295, acc 0.687617\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1000\n",
      "\n",
      "2017-07-01T10:33:52.253912: step 1010, loss 0.539836, acc 0.796875\n",
      "2017-07-01T10:33:53.313731: step 1020, loss 0.479594, acc 0.84375\n",
      "2017-07-01T10:33:54.362521: step 1030, loss 0.490745, acc 0.84375\n",
      "2017-07-01T10:33:55.409305: step 1040, loss 0.525045, acc 0.796875\n",
      "2017-07-01T10:33:56.408964: step 1050, loss 0.448229, acc 0.883333\n",
      "2017-07-01T10:33:57.414670: step 1060, loss 0.511969, acc 0.78125\n",
      "2017-07-01T10:33:58.422320: step 1070, loss 0.464351, acc 0.84375\n",
      "2017-07-01T10:33:59.437019: step 1080, loss 0.477681, acc 0.8125\n",
      "2017-07-01T10:34:00.459773: step 1090, loss 0.429944, acc 0.875\n",
      "2017-07-01T10:34:01.491516: step 1100, loss 0.41096, acc 0.921875\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:34:01.561671: step 1100, loss 0.603617, acc 0.689493\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1100\n",
      "\n",
      "2017-07-01T10:34:03.731077: step 1110, loss 0.544983, acc 0.75\n",
      "2017-07-01T10:34:04.795892: step 1120, loss 0.469, acc 0.828125\n",
      "2017-07-01T10:34:05.776516: step 1130, loss 0.49528, acc 0.84375\n",
      "2017-07-01T10:34:06.787189: step 1140, loss 0.463622, acc 0.828125\n",
      "2017-07-01T10:34:07.785876: step 1150, loss 0.557366, acc 0.75\n",
      "2017-07-01T10:34:08.802551: step 1160, loss 0.440854, acc 0.890625\n",
      "2017-07-01T10:34:09.837301: step 1170, loss 0.485487, acc 0.796875\n",
      "2017-07-01T10:34:10.880107: step 1180, loss 0.43798, acc 0.859375\n",
      "2017-07-01T10:34:11.905837: step 1190, loss 0.489273, acc 0.828125\n",
      "2017-07-01T10:34:12.913485: step 1200, loss 0.479139, acc 0.833333\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:34:12.983671: step 1200, loss 0.593416, acc 0.708255\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1200\n",
      "\n",
      "2017-07-01T10:34:15.149204: step 1210, loss 0.434583, acc 0.875\n",
      "2017-07-01T10:34:16.165908: step 1220, loss 0.405718, acc 0.90625\n",
      "2017-07-01T10:34:17.245995: step 1230, loss 0.422773, acc 0.875\n",
      "2017-07-01T10:34:18.290163: step 1240, loss 0.456117, acc 0.828125\n",
      "2017-07-01T10:34:19.318900: step 1250, loss 0.436187, acc 0.859375\n",
      "2017-07-01T10:34:20.344627: step 1260, loss 0.479941, acc 0.8125\n",
      "2017-07-01T10:34:21.379380: step 1270, loss 0.391403, acc 0.921875\n",
      "2017-07-01T10:34:22.420148: step 1280, loss 0.498078, acc 0.828125\n",
      "2017-07-01T10:34:23.466674: step 1290, loss 0.461365, acc 0.828125\n",
      "2017-07-01T10:34:24.452295: step 1300, loss 0.474488, acc 0.8125\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:34:24.523485: step 1300, loss 0.590842, acc 0.708255\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1300\n",
      "\n",
      "2017-07-01T10:34:26.751633: step 1310, loss 0.484206, acc 0.828125\n",
      "2017-07-01T10:34:27.739251: step 1320, loss 0.479386, acc 0.828125\n",
      "2017-07-01T10:34:28.747968: step 1330, loss 0.459903, acc 0.828125\n",
      "2017-07-01T10:34:29.807754: step 1340, loss 0.511963, acc 0.796875\n",
      "2017-07-01T10:34:30.825463: step 1350, loss 0.410982, acc 0.866667\n",
      "2017-07-01T10:34:31.853226: step 1360, loss 0.465693, acc 0.859375\n",
      "2017-07-01T10:34:32.950114: step 1370, loss 0.433296, acc 0.84375\n",
      "2017-07-01T10:34:33.965815: step 1380, loss 0.371565, acc 0.953125\n",
      "2017-07-01T10:34:34.976541: step 1390, loss 0.407669, acc 0.890625\n",
      "2017-07-01T10:34:36.011257: step 1400, loss 0.38054, acc 0.953125\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:34:36.079437: step 1400, loss 0.594872, acc 0.699812\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1400\n",
      "\n",
      "2017-07-01T10:34:38.179579: step 1410, loss 0.409257, acc 0.890625\n",
      "2017-07-01T10:34:39.231380: step 1420, loss 0.450524, acc 0.84375\n",
      "2017-07-01T10:34:40.258115: step 1430, loss 0.404894, acc 0.890625\n",
      "2017-07-01T10:34:41.289878: step 1440, loss 0.485338, acc 0.84375\n",
      "2017-07-01T10:34:42.295523: step 1450, loss 0.449795, acc 0.859375\n",
      "2017-07-01T10:34:43.361388: step 1460, loss 0.42546, acc 0.890625\n",
      "2017-07-01T10:34:44.376058: step 1470, loss 0.440764, acc 0.875\n",
      "2017-07-01T10:34:45.418836: step 1480, loss 0.413466, acc 0.890625\n",
      "2017-07-01T10:34:46.425540: step 1490, loss 0.428675, acc 0.890625\n",
      "2017-07-01T10:34:47.427176: step 1500, loss 0.419548, acc 0.883333\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:34:47.496358: step 1500, loss 0.586342, acc 0.710131\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1500\n",
      "\n",
      "2017-07-01T10:34:49.746873: step 1510, loss 0.424984, acc 0.890625\n",
      "2017-07-01T10:34:50.776612: step 1520, loss 0.457472, acc 0.828125\n",
      "2017-07-01T10:34:51.832423: step 1530, loss 0.429754, acc 0.875\n",
      "2017-07-01T10:34:52.872186: step 1540, loss 0.40641, acc 0.890625\n",
      "2017-07-01T10:34:53.894967: step 1550, loss 0.427902, acc 0.859375\n",
      "2017-07-01T10:34:54.895632: step 1560, loss 0.417789, acc 0.890625\n",
      "2017-07-01T10:34:55.902277: step 1570, loss 0.487062, acc 0.796875\n",
      "2017-07-01T10:34:56.940066: step 1580, loss 0.418489, acc 0.90625\n",
      "2017-07-01T10:34:57.965779: step 1590, loss 0.373677, acc 0.9375\n",
      "2017-07-01T10:34:58.988499: step 1600, loss 0.393483, acc 0.90625\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:34:59.059690: step 1600, loss 0.583713, acc 0.716698\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1600\n",
      "\n",
      "2017-07-01T10:35:01.219719: step 1610, loss 0.376354, acc 0.953125\n",
      "2017-07-01T10:35:02.249458: step 1620, loss 0.483008, acc 0.8125\n",
      "2017-07-01T10:35:03.269170: step 1630, loss 0.403082, acc 0.90625\n",
      "2017-07-01T10:35:04.278855: step 1640, loss 0.480596, acc 0.828125\n",
      "2017-07-01T10:35:05.294600: step 1650, loss 0.439922, acc 0.85\n",
      "2017-07-01T10:35:06.309256: step 1660, loss 0.386809, acc 0.9375\n",
      "2017-07-01T10:35:07.319945: step 1670, loss 0.40348, acc 0.875\n",
      "2017-07-01T10:35:08.313588: step 1680, loss 0.405258, acc 0.90625\n",
      "2017-07-01T10:35:09.302254: step 1690, loss 0.382914, acc 0.921875\n",
      "2017-07-01T10:35:10.308915: step 1700, loss 0.41532, acc 0.890625\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:35:10.378080: step 1700, loss 0.579837, acc 0.723265\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1700\n",
      "\n",
      "2017-07-01T10:35:12.445026: step 1710, loss 0.433047, acc 0.890625\n",
      "2017-07-01T10:35:13.467747: step 1720, loss 0.444859, acc 0.859375\n",
      "2017-07-01T10:35:14.473422: step 1730, loss 0.367406, acc 0.953125\n",
      "2017-07-01T10:35:15.495140: step 1740, loss 0.411183, acc 0.875\n",
      "2017-07-01T10:35:16.536948: step 1750, loss 0.418962, acc 0.90625\n",
      "2017-07-01T10:35:17.555621: step 1760, loss 0.338044, acc 0.96875\n",
      "2017-07-01T10:35:18.566309: step 1770, loss 0.43669, acc 0.875\n",
      "2017-07-01T10:35:19.569015: step 1780, loss 0.434511, acc 0.890625\n",
      "2017-07-01T10:35:20.584712: step 1790, loss 0.382449, acc 0.921875\n",
      "2017-07-01T10:35:21.598374: step 1800, loss 0.410621, acc 0.916667\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:35:21.667558: step 1800, loss 0.576043, acc 0.730769\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1800\n",
      "\n",
      "2017-07-01T10:35:23.812940: step 1810, loss 0.463813, acc 0.84375\n",
      "2017-07-01T10:35:24.818639: step 1820, loss 0.413981, acc 0.890625\n",
      "2017-07-01T10:35:25.859375: step 1830, loss 0.353669, acc 0.96875\n",
      "2017-07-01T10:35:26.889153: step 1840, loss 0.367382, acc 0.9375\n",
      "2017-07-01T10:35:27.913855: step 1850, loss 0.38567, acc 0.921875\n",
      "2017-07-01T10:35:28.929542: step 1860, loss 0.341804, acc 0.953125\n",
      "2017-07-01T10:35:29.942264: step 1870, loss 0.36645, acc 0.9375\n",
      "2017-07-01T10:35:30.956934: step 1880, loss 0.393178, acc 0.921875\n",
      "2017-07-01T10:35:31.962609: step 1890, loss 0.358683, acc 0.953125\n",
      "2017-07-01T10:35:32.963301: step 1900, loss 0.38115, acc 0.9375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:35:33.031454: step 1900, loss 0.574876, acc 0.734522\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-1900\n",
      "\n",
      "2017-07-01T10:35:35.092619: step 1910, loss 0.435007, acc 0.875\n",
      "2017-07-01T10:35:36.096284: step 1920, loss 0.362199, acc 0.96875\n",
      "2017-07-01T10:35:37.098923: step 1930, loss 0.402282, acc 0.90625\n",
      "2017-07-01T10:35:38.094571: step 1940, loss 0.399775, acc 0.921875\n",
      "2017-07-01T10:35:39.099243: step 1950, loss 0.413501, acc 0.9\n",
      "2017-07-01T10:35:40.110935: step 1960, loss 0.387781, acc 0.921875\n",
      "2017-07-01T10:35:41.111627: step 1970, loss 0.406306, acc 0.90625\n",
      "2017-07-01T10:35:42.120282: step 1980, loss 0.412074, acc 0.90625\n",
      "2017-07-01T10:35:43.178122: step 1990, loss 0.34379, acc 0.96875\n",
      "2017-07-01T10:35:44.179770: step 2000, loss 0.43036, acc 0.890625\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:35:44.247939: step 2000, loss 0.575443, acc 0.722326\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2000\n",
      "\n",
      "2017-07-01T10:35:46.556631: step 2010, loss 0.395006, acc 0.921875\n",
      "2017-07-01T10:35:47.569360: step 2020, loss 0.374188, acc 0.9375\n",
      "2017-07-01T10:35:48.577005: step 2030, loss 0.406845, acc 0.890625\n",
      "2017-07-01T10:35:49.588696: step 2040, loss 0.36534, acc 0.953125\n",
      "2017-07-01T10:35:50.596377: step 2050, loss 0.391002, acc 0.921875\n",
      "2017-07-01T10:35:51.605095: step 2060, loss 0.404828, acc 0.90625\n",
      "2017-07-01T10:35:52.615749: step 2070, loss 0.384054, acc 0.9375\n",
      "2017-07-01T10:35:53.641633: step 2080, loss 0.415825, acc 0.875\n",
      "2017-07-01T10:35:54.655330: step 2090, loss 0.341921, acc 0.984375\n",
      "2017-07-01T10:35:55.674040: step 2100, loss 0.464446, acc 0.85\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:35:55.743224: step 2100, loss 0.580241, acc 0.727955\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2100\n",
      "\n",
      "2017-07-01T10:35:57.839392: step 2110, loss 0.363954, acc 0.953125\n",
      "2017-07-01T10:35:58.837046: step 2120, loss 0.346582, acc 0.96875\n",
      "2017-07-01T10:35:59.846734: step 2130, loss 0.369575, acc 0.9375\n",
      "2017-07-01T10:36:00.863471: step 2140, loss 0.379405, acc 0.921875\n",
      "2017-07-01T10:36:01.896212: step 2150, loss 0.392486, acc 0.90625\n",
      "2017-07-01T10:36:02.907874: step 2160, loss 0.404217, acc 0.890625\n",
      "2017-07-01T10:36:03.928619: step 2170, loss 0.385367, acc 0.921875\n",
      "2017-07-01T10:36:04.951342: step 2180, loss 0.356229, acc 0.96875\n",
      "2017-07-01T10:36:05.957988: step 2190, loss 0.344934, acc 0.984375\n",
      "2017-07-01T10:36:06.976697: step 2200, loss 0.381244, acc 0.9375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:36:07.054914: step 2200, loss 0.577062, acc 0.719512\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2200\n",
      "\n",
      "2017-07-01T10:36:09.262217: step 2210, loss 0.375399, acc 0.9375\n",
      "2017-07-01T10:36:10.279953: step 2220, loss 0.364751, acc 0.9375\n",
      "2017-07-01T10:36:11.297631: step 2230, loss 0.403533, acc 0.90625\n",
      "2017-07-01T10:36:12.315375: step 2240, loss 0.35408, acc 0.9375\n",
      "2017-07-01T10:36:13.325025: step 2250, loss 0.385484, acc 0.916667\n",
      "2017-07-01T10:36:14.338722: step 2260, loss 0.363355, acc 0.96875\n",
      "2017-07-01T10:36:15.344396: step 2270, loss 0.348807, acc 0.96875\n",
      "2017-07-01T10:36:16.367131: step 2280, loss 0.370574, acc 0.9375\n",
      "2017-07-01T10:36:17.378808: step 2290, loss 0.322039, acc 1\n",
      "2017-07-01T10:36:18.377465: step 2300, loss 0.357802, acc 0.9375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:36:18.446648: step 2300, loss 0.574837, acc 0.727955\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2300\n",
      "\n",
      "2017-07-01T10:36:20.478529: step 2310, loss 0.349418, acc 0.96875\n",
      "2017-07-01T10:36:21.490260: step 2320, loss 0.346933, acc 0.96875\n",
      "2017-07-01T10:36:22.514983: step 2330, loss 0.389359, acc 0.921875\n",
      "2017-07-01T10:36:23.533686: step 2340, loss 0.341946, acc 0.96875\n",
      "2017-07-01T10:36:24.549393: step 2350, loss 0.371782, acc 0.9375\n",
      "2017-07-01T10:36:25.569072: step 2360, loss 0.335945, acc 0.984375\n",
      "2017-07-01T10:36:26.591793: step 2370, loss 0.376247, acc 0.921875\n",
      "2017-07-01T10:36:27.619593: step 2380, loss 0.389962, acc 0.921875\n",
      "2017-07-01T10:36:28.632249: step 2390, loss 0.381657, acc 0.9375\n",
      "2017-07-01T10:36:29.638949: step 2400, loss 0.340653, acc 0.983333\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:36:29.708159: step 2400, loss 0.576065, acc 0.727955\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2400\n",
      "\n",
      "2017-07-01T10:36:31.894330: step 2410, loss 0.363246, acc 0.953125\n",
      "2017-07-01T10:36:32.903013: step 2420, loss 0.364245, acc 0.9375\n",
      "2017-07-01T10:36:33.921722: step 2430, loss 0.3343, acc 0.984375\n",
      "2017-07-01T10:36:34.954470: step 2440, loss 0.361444, acc 0.953125\n",
      "2017-07-01T10:36:35.974182: step 2450, loss 0.329239, acc 0.984375\n",
      "2017-07-01T10:36:36.989891: step 2460, loss 0.445064, acc 0.859375\n",
      "2017-07-01T10:36:38.003580: step 2470, loss 0.386216, acc 0.9375\n",
      "2017-07-01T10:36:39.011268: step 2480, loss 0.339535, acc 0.96875\n",
      "2017-07-01T10:36:40.029987: step 2490, loss 0.395512, acc 0.90625\n",
      "2017-07-01T10:36:41.057705: step 2500, loss 0.383568, acc 0.9375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:36:41.129897: step 2500, loss 0.574884, acc 0.725141\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2500\n",
      "\n",
      "2017-07-01T10:36:43.374312: step 2510, loss 0.365758, acc 0.953125\n",
      "2017-07-01T10:36:44.395996: step 2520, loss 0.373341, acc 0.953125\n",
      "2017-07-01T10:36:45.419719: step 2530, loss 0.336472, acc 0.984375\n",
      "2017-07-01T10:36:46.439432: step 2540, loss 0.348033, acc 0.96875\n",
      "2017-07-01T10:36:47.456136: step 2550, loss 0.328073, acc 0.983333\n",
      "2017-07-01T10:36:48.484872: step 2560, loss 0.331029, acc 0.984375\n",
      "2017-07-01T10:36:49.499572: step 2570, loss 0.346202, acc 0.96875\n",
      "2017-07-01T10:36:50.504275: step 2580, loss 0.334339, acc 0.984375\n",
      "2017-07-01T10:36:51.529006: step 2590, loss 0.338842, acc 0.984375\n",
      "2017-07-01T10:36:52.547680: step 2600, loss 0.331423, acc 0.984375\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:36:52.618881: step 2600, loss 0.574863, acc 0.728893\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2600\n",
      "\n",
      "2017-07-01T10:36:54.693748: step 2610, loss 0.350774, acc 0.953125\n",
      "2017-07-01T10:36:55.713461: step 2620, loss 0.34965, acc 0.953125\n",
      "2017-07-01T10:36:56.732200: step 2630, loss 0.357182, acc 0.96875\n",
      "2017-07-01T10:36:57.752923: step 2640, loss 0.361011, acc 0.921875\n",
      "2017-07-01T10:36:58.774643: step 2650, loss 0.350866, acc 0.96875\n",
      "2017-07-01T10:36:59.780316: step 2660, loss 0.368551, acc 0.953125\n",
      "2017-07-01T10:37:00.780009: step 2670, loss 0.344794, acc 0.953125\n",
      "2017-07-01T10:37:01.787657: step 2680, loss 0.361908, acc 0.953125\n",
      "2017-07-01T10:37:02.794333: step 2690, loss 0.341989, acc 0.984375\n",
      "2017-07-01T10:37:03.807107: step 2700, loss 0.31528, acc 1\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:37:03.876297: step 2700, loss 0.579503, acc 0.724203\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2700\n",
      "\n",
      "2017-07-01T10:37:05.994234: step 2710, loss 0.340915, acc 0.96875\n",
      "2017-07-01T10:37:06.996907: step 2720, loss 0.344756, acc 0.96875\n",
      "2017-07-01T10:37:07.991546: step 2730, loss 0.33369, acc 0.984375\n",
      "2017-07-01T10:37:09.003268: step 2740, loss 0.31929, acc 1\n",
      "2017-07-01T10:37:10.019942: step 2750, loss 0.331746, acc 0.984375\n",
      "2017-07-01T10:37:11.034669: step 2760, loss 0.353704, acc 0.96875\n",
      "2017-07-01T10:37:12.039313: step 2770, loss 0.337309, acc 0.96875\n",
      "2017-07-01T10:37:13.048003: step 2780, loss 0.333914, acc 0.984375\n",
      "2017-07-01T10:37:14.064701: step 2790, loss 0.36784, acc 0.921875\n",
      "2017-07-01T10:37:15.066371: step 2800, loss 0.324945, acc 1\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:37:15.136552: step 2800, loss 0.573902, acc 0.733584\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2800\n",
      "\n",
      "2017-07-01T10:37:17.075880: step 2810, loss 0.335675, acc 0.984375\n",
      "2017-07-01T10:37:18.076542: step 2820, loss 0.359467, acc 0.953125\n",
      "2017-07-01T10:37:19.079243: step 2830, loss 0.386529, acc 0.9375\n",
      "2017-07-01T10:37:20.076862: step 2840, loss 0.371979, acc 0.9375\n",
      "2017-07-01T10:37:21.078556: step 2850, loss 0.316938, acc 1\n",
      "2017-07-01T10:37:22.082234: step 2860, loss 0.319891, acc 1\n",
      "2017-07-01T10:37:23.087902: step 2870, loss 0.349129, acc 0.96875\n",
      "2017-07-01T10:37:24.081515: step 2880, loss 0.375307, acc 0.9375\n",
      "2017-07-01T10:37:25.068200: step 2890, loss 0.319795, acc 1\n",
      "2017-07-01T10:37:26.063788: step 2900, loss 0.349752, acc 0.953125\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:37:26.132972: step 2900, loss 0.581737, acc 0.716698\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-2900\n",
      "\n",
      "2017-07-01T10:37:28.209692: step 2910, loss 0.347791, acc 0.96875\n",
      "2017-07-01T10:37:29.204337: step 2920, loss 0.345461, acc 0.96875\n",
      "2017-07-01T10:37:30.207040: step 2930, loss 0.345496, acc 0.96875\n",
      "2017-07-01T10:37:31.209709: step 2940, loss 0.343757, acc 0.984375\n",
      "2017-07-01T10:37:32.213342: step 2950, loss 0.354124, acc 0.953125\n",
      "2017-07-01T10:37:33.205982: step 2960, loss 0.342052, acc 0.96875\n",
      "2017-07-01T10:37:34.208667: step 2970, loss 0.33255, acc 0.984375\n",
      "2017-07-01T10:37:35.221343: step 2980, loss 0.379844, acc 0.9375\n",
      "2017-07-01T10:37:36.224047: step 2990, loss 0.334321, acc 0.984375\n",
      "2017-07-01T10:37:37.224671: step 3000, loss 0.350822, acc 0.966667\n",
      "\n",
      "Evaluation:\n",
      "2017-07-01T10:37:37.294858: step 3000, loss 0.573427, acc 0.732645\n",
      "\n",
      "Saved model checkpoint to D:\\Tensorflow\\cnn-text-classification-tf-master\\runs\\1498876300\\checkpoints\\model-3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "# ==================================================\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    session_conf = tf.ConfigProto(\n",
    "      allow_soft_placement=FLAGS.allow_soft_placement,\n",
    "      log_device_placement=FLAGS.log_device_placement)\n",
    "    sess = tf.Session(config=session_conf)\n",
    "    with sess.as_default():\n",
    "        cnn = TextCNN(\n",
    "            sequence_length=x_train.shape[1],\n",
    "            num_classes=y_train.shape[1],\n",
    "            vocab_size=len(vocab_processor.vocabulary_),\n",
    "            embedding_size=FLAGS.embedding_dim,\n",
    "            filter_sizes=list(map(int, FLAGS.filter_sizes.split(\",\"))),\n",
    "            num_filters=FLAGS.num_filters,\n",
    "            l2_reg_lambda=FLAGS.l2_reg_lambda)\n",
    "\n",
    "        # Define Training procedure\n",
    "        global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "        optimizer = tf.train.AdamOptimizer(1e-3)\n",
    "        # 计算梯度\n",
    "        grads_and_vars = optimizer.compute_gradients(cnn.loss)\n",
    "        # 将计算出的梯度应用到变量上，是函数minimize()的第二部分，\n",
    "        # 返回一个应用指定的梯度的操作Operation，对global_step做自增操作\n",
    "        train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)\n",
    "\n",
    "        # Keep track of gradient values and sparsity (optional)\n",
    "        # 保存变量的梯度值\n",
    "        grad_summaries = []\n",
    "        for g, v in grads_and_vars:\n",
    "            if g is not None:\n",
    "                grad_hist_summary = tf.summary.histogram(\"{}/grad/hist\".format(v.name), g)\n",
    "                sparsity_summary = tf.summary.scalar(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                grad_summaries.append(grad_hist_summary)\n",
    "                grad_summaries.append(sparsity_summary)\n",
    "        grad_summaries_merged = tf.summary.merge(grad_summaries)\n",
    "\n",
    "        # Output directory for models and summaries\n",
    "        # 定义输出路径\n",
    "        timestamp = str(int(time.time()))\n",
    "        out_dir = os.path.abspath(os.path.join(os.path.curdir, \"runs\", timestamp))\n",
    "        print(\"Writing to {}\\n\".format(out_dir))\n",
    "\n",
    "        # Summaries for loss and accuracy\n",
    "        loss_summary = tf.summary.scalar(\"loss\", cnn.loss)\n",
    "        acc_summary = tf.summary.scalar(\"accuracy\", cnn.accuracy)\n",
    "\n",
    "        # Train Summaries\n",
    "        train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged])\n",
    "        train_summary_dir = os.path.join(out_dir, \"summaries\", \"train\")\n",
    "        train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)\n",
    "\n",
    "        # Dev summaries\n",
    "        dev_summary_op = tf.summary.merge([loss_summary, acc_summary])\n",
    "        dev_summary_dir = os.path.join(out_dir, \"summaries\", \"dev\")\n",
    "        dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)\n",
    "\n",
    "        # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it\n",
    "        checkpoint_dir = os.path.abspath(os.path.join(out_dir, \"checkpoints\"))\n",
    "        checkpoint_prefix = os.path.join(checkpoint_dir, \"model\")\n",
    "        if not os.path.exists(checkpoint_dir):\n",
    "            os.makedirs(checkpoint_dir)\n",
    "        # 保存模型，最多保存5个模型\n",
    "        saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)\n",
    "\n",
    "        # Write vocabulary\n",
    "        vocab_processor.save(os.path.join(out_dir, \"vocab\"))\n",
    "\n",
    "        # Initialize all variables\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        def train_step(x_batch, y_batch):\n",
    "            \"\"\"\n",
    "            A single training step\n",
    "            \"\"\"\n",
    "            feed_dict = {\n",
    "              cnn.input_x: x_batch,\n",
    "              cnn.input_y: y_batch,\n",
    "              cnn.dropout_keep_prob: FLAGS.dropout_keep_prob\n",
    "            }\n",
    "            _, step, summaries, loss, accuracy = sess.run(\n",
    "                [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],\n",
    "                feed_dict)\n",
    "            time_str = datetime.datetime.now().isoformat()\n",
    "            if (step%10==0):\n",
    "                print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "            train_summary_writer.add_summary(summaries, step)\n",
    "\n",
    "        def dev_step(x_batch, y_batch, writer=None):\n",
    "            \"\"\"\n",
    "            Evaluates model on a dev set\n",
    "            \"\"\"\n",
    "            feed_dict = {\n",
    "              cnn.input_x: x_batch,\n",
    "              cnn.input_y: y_batch,\n",
    "              cnn.dropout_keep_prob: 1.0\n",
    "            }\n",
    "            step, summaries, loss, accuracy = sess.run(\n",
    "                [global_step, dev_summary_op, cnn.loss, cnn.accuracy],\n",
    "                feed_dict)\n",
    "            time_str = datetime.datetime.now().isoformat()\n",
    "            print(\"{}: step {}, loss {:g}, acc {:g}\".format(time_str, step, loss, accuracy))\n",
    "            if writer:\n",
    "                writer.add_summary(summaries, step)\n",
    "\n",
    "        # Generate batches\n",
    "        batches = data_helpers.batch_iter(\n",
    "            list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs)\n",
    "        # Training loop. For each batch...\n",
    "        for batch in batches:\n",
    "            x_batch, y_batch = zip(*batch)\n",
    "            train_step(x_batch, y_batch)\n",
    "            current_step = tf.train.global_step(sess, global_step)\n",
    "            # 测试\n",
    "            if current_step % FLAGS.evaluate_every == 0:\n",
    "                print(\"\\nEvaluation:\")\n",
    "                dev_step(x_dev, y_dev, writer=dev_summary_writer)\n",
    "                print(\"\")\n",
    "            # 保存模型\n",
    "            if current_step % FLAGS.checkpoint_every == 0:\n",
    "                path = saver.save(sess, checkpoint_prefix, global_step=current_step)\n",
    "                print(\"Saved model checkpoint to {}\\n\".format(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
